﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using System.Text.RegularExpressions;
using System.Collections;
using System.ComponentModel;

namespace SearchEngine
{
    class Crawlerpro
    {
        public string startRef = " ";
        public string endRef =  "";

        public void Crawl()
        {
            string html = Download(startRef);
            endRef=Parse(html);

        }

        public string Download(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                //string fileName = "结果";
                //File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                //Console.WriteLine(ex.Message);
                //显示错误的URL信息
                return "";
            }
        }

        public string Parse(string html)
        {


            //提取前200字
            string x = @"[\u4E00-\u9FFF]+";
            MatchCollection Matches = Regex.Matches
            (html, x, RegexOptions.IgnoreCase);
            string tem = "";
            foreach (Match NextMatch in Matches)
            {
                if (!NextMatch.Value.Contains("百度"))
                {
                    tem = tem + NextMatch.Value;
                }

            }

            if (tem.Length > 200)
            {
                return tem.Substring(0, 200);
            }
            else
            {
                return tem;
            }
            /*string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);*/
            /*foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (strRef.Length == 0) continue;
                if (!strRef.Contains(".html")) continue;//不是html页不加入爬取队列中

                //相对地址转换为绝对地址
                if (strRef[0] == '/')
                {
                    strRef = new Uri(new Uri(startRef), strRef).ToString();
                }


                if (urls[strRef] == null) urls[strRef] = false;
            }*/
        }
        /*static void Main(string[] args)
        {
            Console.WriteLine("Hello World!");
        }*/
    }
}
